


<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  <meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />

  <meta name="viewport" content="width=device-width, initial-scale=1.0">

  <title>Torch-TensorRT Getting Started - CitriNet &mdash; Torch-TensorRT master documentation</title>















  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/collapsible-lists/css/tree_view.css" type="text/css" />
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
  <!-- Google Analytics -->

  <!-- End Google Analytics -->



  <script src="../_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
  <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.15.2/css/all.css" integrity="sha384-vSIIfh2YWi9wW0r9iZe7RJPrKwp6bG+s9QZMoITbCckVJqGCCRhc+ccxNcdpHuYu" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li>
            <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="resource-option with-down-orange-arrow">
                Docs
              </a>
              <div class="resources-dropdown-menu">
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/docs/stable/index.html">
                  <span class="dropdown-title">PyTorch</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/audio/stable/index.html">
                  <span class="dropdown-title">torchaudio</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/text/stable/index.html">
                  <span class="dropdown-title">torchtext</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/vision/stable/index.html">
                  <span class="dropdown-title">torchvision</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/torchrec">
                  <span class="dropdown-title">TorchRec</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/data">
                  <span class="dropdown-title">TorchData</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/serve/">
                  <span class="dropdown-title">TorchServe</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/torchx/">
                  <span class="dropdown-title">TorchX</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/xla">
                  <span class="dropdown-title">PyTorch on XLA Devices</span>
                  <p></p>
                </a>
            </div>
          </li>

          <li>
            <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="resource-option with-down-arrow">
                Resources
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/features">
                  <span class="dropdown-title">About</span>
                  <p>Learn about PyTorch’s features and capabilities</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/#community-module">
                  <span class="dropdown-title">Community</span>
                  <p>Join the PyTorch developer community to contribute, learn, and get your questions answered.</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/resources">
                  <span class="dropdown-title">Developer Resources</span>
                  <p>Find resources and get questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://discuss.pytorch.org/" target="_blank">
                  <span class="dropdown-title">Forums</span>
                  <p>A place to discuss PyTorch code, issues, install, research</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/hub">
                  <span class="dropdown-title">Models (Beta)</span>
                  <p>Discover, publish, and reuse pre-trained models</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">GitHub</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>
  </div>
</div>

<body class="pytorch-body">





    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">





                <div class="version">
                  master (1.2.0)
                </div>









<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>


          </div>







              <p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../getting_started/installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../getting_started/getting_started_with_python_api.html">Using Torch-TensorRT in Python</a></li>
<li class="toctree-l1"><a class="reference internal" href="../getting_started/getting_started_with_cpp_api.html">Using Torch-TensorRT in  C++</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Tutorials</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/creating_torchscript_module_in_python.html">Creating a TorchScript Module</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/creating_torchscript_module_in_python.html#working-with-torchscript-in-python">Working with TorchScript in Python</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/creating_torchscript_module_in_python.html#saving-torchscript-module-to-disk">Saving TorchScript Module to Disk</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/getting_started_with_fx_path.html">Torch-TensorRT (FX Frontend) User Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/ptq.html">Post Training Quantization (PTQ)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/runtime.html">Deploying Torch-TensorRT Programs</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/serving_torch_tensorrt_with_triton.html">Serving a Torch-TensorRT model with Triton</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/use_from_pytorch.html">Using Torch-TensorRT Directly From PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/using_dla.html">DLA</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/notebooks.html">Example notebooks</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Python API Documenation</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../py_api/torch_tensorrt.html">torch_tensorrt</a></li>
<li class="toctree-l1"><a class="reference internal" href="../py_api/logging.html">torch_tensorrt.logging</a></li>
<li class="toctree-l1"><a class="reference internal" href="../py_api/ptq.html">torch_tensorrt.ptq</a></li>
<li class="toctree-l1"><a class="reference internal" href="../py_api/ts.html">torch_tensorrt.ts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../py_api/fx.html">torch_tensorrt.fx</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">C++ API Documenation</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../_cpp_api/torch_tensort_cpp.html">Torch-TensorRT C++ API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../_cpp_api/namespace_torch_tensorrt.html">Namespace torch_tensorrt</a></li>
<li class="toctree-l1"><a class="reference internal" href="../_cpp_api/namespace_torch_tensorrt__logging.html">Namespace torch_tensorrt::logging</a></li>
<li class="toctree-l1"><a class="reference internal" href="../_cpp_api/namespace_torch_tensorrt__torchscript.html">Namespace torch_tensorrt::torchscript</a></li>
<li class="toctree-l1"><a class="reference internal" href="../_cpp_api/namespace_torch_tensorrt__ptq.html">Namespace torch_tensorrt::ptq</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">CLI Documenation</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../cli/torchtrtc.html">torchtrtc</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Contributor Documentation</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../contributors/system_overview.html">System Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contributors/writing_converters.html">Writing Converters</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contributors/useful_links.html">Useful Links for Torch-TensorRT Development</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Indices</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../indices/supported_ops.html">Operators Supported</a></li>
</ul>



        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">

      <li>
        <a href="../index.html">

            Docs

        </a> &gt;
      </li>


      <li>Torch-TensorRT Getting Started - CitriNet</li>


      <li class="pytorch-breadcrumbs-aside">


            <a href="../_sources/_notebooks/CitriNet-example.ipynb.txt" rel="nofollow"><img src="../_static/images/view-page-source-icon.svg"></a>


      </li>

  </ul>


</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">



          <div class="rst-content">

            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">


<style>
/* CSS for nbsphinx extension */

/* remove conflicting styling from Sphinx themes */
div.nbinput.container div.prompt *,
div.nboutput.container div.prompt *,
div.nbinput.container div.input_area pre,
div.nboutput.container div.output_area pre,
div.nbinput.container div.input_area .highlight,
div.nboutput.container div.output_area .highlight {
    border: none;
    padding: 0;
    margin: 0;
    box-shadow: none;
}

div.nbinput.container > div[class*=highlight],
div.nboutput.container > div[class*=highlight] {
    margin: 0;
}

div.nbinput.container div.prompt *,
div.nboutput.container div.prompt * {
    background: none;
}

div.nboutput.container div.output_area .highlight,
div.nboutput.container div.output_area pre {
    background: unset;
}

div.nboutput.container div.output_area div.highlight {
    color: unset;  /* override Pygments text color */
}

/* avoid gaps between output lines */
div.nboutput.container div[class*=highlight] pre {
    line-height: normal;
}

/* input/output containers */
div.nbinput.container,
div.nboutput.container {
    display: -webkit-flex;
    display: flex;
    align-items: flex-start;
    margin: 0;
    width: 100%;
}
@media (max-width: 540px) {
    div.nbinput.container,
    div.nboutput.container {
        flex-direction: column;
    }
}

/* input container */
div.nbinput.container {
    padding-top: 5px;
}

/* last container */
div.nblast.container {
    padding-bottom: 5px;
}

/* input prompt */
div.nbinput.container div.prompt pre {
    color: #307FC1;
}

/* output prompt */
div.nboutput.container div.prompt pre {
    color: #BF5B3D;
}

/* all prompts */
div.nbinput.container div.prompt,
div.nboutput.container div.prompt {
    width: 4.5ex;
    padding-top: 5px;
    position: relative;
    user-select: none;
}

div.nbinput.container div.prompt > div,
div.nboutput.container div.prompt > div {
    position: absolute;
    right: 0;
    margin-right: 0.3ex;
}

@media (max-width: 540px) {
    div.nbinput.container div.prompt,
    div.nboutput.container div.prompt {
        width: unset;
        text-align: left;
        padding: 0.4em;
    }
    div.nboutput.container div.prompt.empty {
        padding: 0;
    }

    div.nbinput.container div.prompt > div,
    div.nboutput.container div.prompt > div {
        position: unset;
    }
}

/* disable scrollbars on prompts */
div.nbinput.container div.prompt pre,
div.nboutput.container div.prompt pre {
    overflow: hidden;
}

/* input/output area */
div.nbinput.container div.input_area,
div.nboutput.container div.output_area {
    -webkit-flex: 1;
    flex: 1;
    overflow: auto;
}
@media (max-width: 540px) {
    div.nbinput.container div.input_area,
    div.nboutput.container div.output_area {
        width: 100%;
    }
}

/* input area */
div.nbinput.container div.input_area {
    border: 1px solid #e0e0e0;
    border-radius: 2px;
    /*background: #f5f5f5;*/
}

/* override MathJax center alignment in output cells */
div.nboutput.container div[class*=MathJax] {
    text-align: left !important;
}

/* override sphinx.ext.imgmath center alignment in output cells */
div.nboutput.container div.math p {
    text-align: left;
}

/* standard error */
div.nboutput.container div.output_area.stderr {
    background: #fdd;
}

/* ANSI colors */
.ansi-black-fg { color: #3E424D; }
.ansi-black-bg { background-color: #3E424D; }
.ansi-black-intense-fg { color: #282C36; }
.ansi-black-intense-bg { background-color: #282C36; }
.ansi-red-fg { color: #E75C58; }
.ansi-red-bg { background-color: #E75C58; }
.ansi-red-intense-fg { color: #B22B31; }
.ansi-red-intense-bg { background-color: #B22B31; }
.ansi-green-fg { color: #00A250; }
.ansi-green-bg { background-color: #00A250; }
.ansi-green-intense-fg { color: #007427; }
.ansi-green-intense-bg { background-color: #007427; }
.ansi-yellow-fg { color: #DDB62B; }
.ansi-yellow-bg { background-color: #DDB62B; }
.ansi-yellow-intense-fg { color: #B27D12; }
.ansi-yellow-intense-bg { background-color: #B27D12; }
.ansi-blue-fg { color: #208FFB; }
.ansi-blue-bg { background-color: #208FFB; }
.ansi-blue-intense-fg { color: #0065CA; }
.ansi-blue-intense-bg { background-color: #0065CA; }
.ansi-magenta-fg { color: #D160C4; }
.ansi-magenta-bg { background-color: #D160C4; }
.ansi-magenta-intense-fg { color: #A03196; }
.ansi-magenta-intense-bg { background-color: #A03196; }
.ansi-cyan-fg { color: #60C6C8; }
.ansi-cyan-bg { background-color: #60C6C8; }
.ansi-cyan-intense-fg { color: #258F8F; }
.ansi-cyan-intense-bg { background-color: #258F8F; }
.ansi-white-fg { color: #C5C1B4; }
.ansi-white-bg { background-color: #C5C1B4; }
.ansi-white-intense-fg { color: #A1A6B2; }
.ansi-white-intense-bg { background-color: #A1A6B2; }

.ansi-default-inverse-fg { color: #FFFFFF; }
.ansi-default-inverse-bg { background-color: #000000; }

.ansi-bold { font-weight: bold; }
.ansi-underline { text-decoration: underline; }


div.nbinput.container div.input_area div[class*=highlight] > pre,
div.nboutput.container div.output_area div[class*=highlight] > pre,
div.nboutput.container div.output_area div[class*=highlight].math,
div.nboutput.container div.output_area.rendered_html,
div.nboutput.container div.output_area > div.output_javascript,
div.nboutput.container div.output_area:not(.rendered_html) > img{
    padding: 5px;
    margin: 0;
}

/* fix copybtn overflow problem in chromium (needed for 'sphinx_copybutton') */
div.nbinput.container div.input_area > div[class^='highlight'],
div.nboutput.container div.output_area > div[class^='highlight']{
    overflow-y: hidden;
}

/* hide copybtn icon on prompts (needed for 'sphinx_copybutton') */
.prompt .copybtn {
    display: none;
}

/* Some additional styling taken form the Jupyter notebook CSS */
div.rendered_html table {
  border: none;
  border-collapse: collapse;
  border-spacing: 0;
  color: black;
  font-size: 12px;
  table-layout: fixed;
}
div.rendered_html thead {
  border-bottom: 1px solid black;
  vertical-align: bottom;
}
div.rendered_html tr,
div.rendered_html th,
div.rendered_html td {
  text-align: right;
  vertical-align: middle;
  padding: 0.5em 0.5em;
  line-height: normal;
  white-space: normal;
  max-width: none;
  border: none;
}
div.rendered_html th {
  font-weight: bold;
}
div.rendered_html tbody tr:nth-child(odd) {
  background: #f5f5f5;
}
div.rendered_html tbody tr:hover {
  background: rgba(66, 165, 245, 0.2);
}
</style>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[1]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span># Copyright 2019 NVIDIA Corporation. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an &quot;AS IS&quot; BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
</pre></div>
</div>
</div>
<p><img alt="198e4deb2921411abba2a37241da72ab" src="http://developer.download.nvidia.com/compute/machine-learning/frameworks/nvidia_logo.png" /></p>
<section id="Torch-TensorRT-Getting-Started---CitriNet">
<h1>Torch-TensorRT Getting Started - CitriNet<a class="headerlink" href="#Torch-TensorRT-Getting-Started---CitriNet" title="Permalink to this headline">¶</a></h1>
<section id="Overview">
<h2>Overview<a class="headerlink" href="#Overview" title="Permalink to this headline">¶</a></h2>
<p><a class="reference external" href="https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/asr/models.html#citrinet">Citrinet</a> is an acoustic model used for the speech to text recognition task. It is a version of <a class="reference external" href="https://arxiv.org/pdf/1910.10261.pdf">QuartzNet</a> that extends <a class="reference external" href="https://arxiv.org/pdf/2005.03191.pdf">ContextNet</a>, utilizing subword encoding (via Word Piece tokenization) and Squeeze-and-Excitation(SE) mechanism and are therefore smaller than QuartzNet models.</p>
<p>CitriNet models take in audio segments and transcribe them to letter, byte pair, or word piece sequences.</p>
<p><img alt="alt" class="no-scaled-link" src="https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/_images/jasper_vertical.png" style="width: 50%;" /></p>
<section id="Learning-objectives">
<h3>Learning objectives<a class="headerlink" href="#Learning-objectives" title="Permalink to this headline">¶</a></h3>
<p>This notebook demonstrates the steps for optimizing a pretrained CitriNet model with Torch-TensorRT, and running it to test the speedup obtained.</p>
</section>
</section>
<section id="Content">
<h2>Content<a class="headerlink" href="#Content" title="Permalink to this headline">¶</a></h2>
<ol class="arabic simple">
<li><p><a class="reference external" href="#1">Requirements</a></p></li>
<li><p><a class="reference external" href="#2">Download Citrinet model</a></p></li>
<li><p><a class="reference external" href="#3">Create Torch-TensorRT modules</a></p></li>
<li><p><a class="reference external" href="#4">Benchmark Torch-TensorRT models</a></p></li>
<li><p><a class="reference external" href="#5">Conclusion</a></p></li>
</ol>
<p>## 1. Requirements</p>
<p>Follow the steps in <a class="reference external" href="README.md">README</a> to prepare a Docker container, within which you can run this notebook. This notebook assumes that you are within a Jupyter environment in a docker container with Torch-TensorRT installed, such as an NGC monthly release of <code class="docutils literal notranslate"><span class="pre">nvcr.io/nvidia/pytorch:&lt;yy.mm&gt;-py3</span></code> (where <code class="docutils literal notranslate"><span class="pre">yy</span></code> indicates the last two numbers of a calendar year, and <code class="docutils literal notranslate"><span class="pre">mm</span></code> indicates the month in two-digit numerical form)</p>
<p>Now that you are in the docker, the next step is to install the required dependencies.</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[2]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span># Install dependencies
!pip install wget
!apt-get update &amp;&amp; DEBIAN_FRONTEND=noninteractive  apt-get install -y libsndfile1 ffmpeg
!pip install Cython

## Install NeMo
!pip install nemo_toolkit[all]==1.5.1
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com
Requirement already satisfied: wget in /opt/conda/lib/python3.8/site-packages (3.2)
<span class="ansi-yellow-fg">WARNING: Running pip as the &#39;root&#39; user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv</span>
Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease
Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease
Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease
Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease
Reading package lists... Done
Reading package lists... Done
Building dependency tree
Reading state information... Done
libsndfile1 is already the newest version (1.0.28-7ubuntu0.1).
ffmpeg is already the newest version (7:4.2.4-1ubuntu0.1).
0 upgraded, 0 newly installed, 0 to remove and 22 not upgraded.
Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com
Requirement already satisfied: Cython in /opt/conda/lib/python3.8/site-packages (0.29.28)
<span class="ansi-yellow-fg">WARNING: Running pip as the &#39;root&#39; user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv</span>
Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com
Requirement already satisfied: nemo_toolkit[all]==1.5.1 in /opt/conda/lib/python3.8/site-packages (1.5.1)
Requirement already satisfied: numpy&gt;=1.18.2 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (1.22.3)
Requirement already satisfied: onnx&gt;=1.7.0 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (1.10.1)
Requirement already satisfied: python-dateutil in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (2.8.2)
Requirement already satisfied: tqdm&gt;=4.41.0 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (4.63.0)
Requirement already satisfied: sentencepiece&lt;1.0.0 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.1.96)
Requirement already satisfied: wget in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (3.2)
Requirement already satisfied: numba in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.53.1)
Requirement already satisfied: torch in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (1.12.0a0+2c916ef)
Requirement already satisfied: unidecode in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (1.3.4)
Requirement already satisfied: frozendict in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (2.3.2)
Requirement already satisfied: wrapt in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (1.14.0)
Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.24.2)
Requirement already satisfied: ruamel.yaml in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.17.21)
Requirement already satisfied: pesq in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.0.3)
Requirement already satisfied: torchvision in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.13.0a0)
Requirement already satisfied: gdown in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (4.4.0)
Requirement already satisfied: editdistance in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.6.0)
Requirement already satisfied: boto3 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (1.21.45)
Requirement already satisfied: isort[requirements]&lt;5 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (4.3.21)
Requirement already satisfied: hydra-core&gt;=1.1.0 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (1.1.2)
Requirement already satisfied: youtokentome&gt;=1.0.5 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (1.0.6)
Requirement already satisfied: pytorch-lightning&gt;=1.5.0 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (1.6.1)
Requirement already satisfied: jieba in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.42.1)
Requirement already satisfied: fasttext in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.9.2)
Requirement already satisfied: soundfile in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.10.3.post1)
Requirement already satisfied: kaldiio in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (2.17.2)
Requirement already satisfied: pangu in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (4.0.6.1)
Requirement already satisfied: kaldi-python-io in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (1.2.2)
Requirement already satisfied: parameterized in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.8.1)
Requirement already satisfied: h5py in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (3.6.0)
Requirement already satisfied: rapidfuzz in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (2.0.10)
Requirement already satisfied: marshmallow in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (3.15.0)
Requirement already satisfied: opencc in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (1.1.3)
Requirement already satisfied: braceexpand in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.1.7)
Requirement already satisfied: omegaconf&gt;=2.1.0 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (2.1.2)
Requirement already satisfied: sphinx in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (4.4.0)
Requirement already satisfied: pillow in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (9.0.0)
Requirement already satisfied: wordninja==2.0.0 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (2.0.0)
Requirement already satisfied: torch-stft in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.1.4)
Requirement already satisfied: sox in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (1.4.1)
Requirement already satisfied: librosa in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.9.1)
Requirement already satisfied: regex in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (2022.3.15)
Requirement already satisfied: sacrebleu[ja] in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (2.0.0)
Requirement already satisfied: black==19.10b0 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (19.10b0)
Requirement already satisfied: pydub in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.25.1)
Requirement already satisfied: sphinxcontrib-bibtex in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (2.4.2)
Requirement already satisfied: inflect in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (5.5.2)
Requirement already satisfied: pyannote.core in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (4.4)
Requirement already satisfied: packaging in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (21.3)
Requirement already satisfied: kaldi-io in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.9.4)
Requirement already satisfied: pyannote.metrics in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (3.2)
Requirement already satisfied: g2p-en in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (2.1.0)
Requirement already satisfied: matplotlib in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (3.5.1)
Requirement already satisfied: torchmetrics&gt;=0.4.1rc0 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.8.0)
Requirement already satisfied: nltk&gt;=3.6.5 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (3.7)
Requirement already satisfied: pyyaml&lt;6 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (5.4.1)
Requirement already satisfied: scipy in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (1.6.3)
Requirement already satisfied: ipywidgets in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (7.7.0)
Requirement already satisfied: pytest in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (6.2.5)
Requirement already satisfied: pandas in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (1.3.5)
Requirement already satisfied: pytest-runner in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (6.0.0)
Requirement already satisfied: transformers&gt;=4.0.1 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (4.18.0)
Requirement already satisfied: sacremoses&gt;=0.0.43 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.0.49)
Requirement already satisfied: pystoi in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.3.3)
Requirement already satisfied: attrdict in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (2.0.1)
Requirement already satisfied: webdataset&lt;=0.1.62,&gt;=0.1.48 in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.1.62)
Requirement already satisfied: wandb in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.12.15)
Requirement already satisfied: pypinyin in /opt/conda/lib/python3.8/site-packages (from nemo_toolkit[all]==1.5.1) (0.46.0)
Requirement already satisfied: attrs&gt;=18.1.0 in /opt/conda/lib/python3.8/site-packages (from black==19.10b0-&gt;nemo_toolkit[all]==1.5.1) (21.4.0)
Requirement already satisfied: appdirs in /opt/conda/lib/python3.8/site-packages (from black==19.10b0-&gt;nemo_toolkit[all]==1.5.1) (1.4.4)
Requirement already satisfied: typed-ast&gt;=1.4.0 in /opt/conda/lib/python3.8/site-packages (from black==19.10b0-&gt;nemo_toolkit[all]==1.5.1) (1.5.3)
Requirement already satisfied: pathspec&lt;1,&gt;=0.6 in /opt/conda/lib/python3.8/site-packages (from black==19.10b0-&gt;nemo_toolkit[all]==1.5.1) (0.9.0)
Requirement already satisfied: click&gt;=6.5 in /opt/conda/lib/python3.8/site-packages (from black==19.10b0-&gt;nemo_toolkit[all]==1.5.1) (8.0.4)
Requirement already satisfied: toml&gt;=0.9.4 in /opt/conda/lib/python3.8/site-packages (from black==19.10b0-&gt;nemo_toolkit[all]==1.5.1) (0.10.2)
Requirement already satisfied: antlr4-python3-runtime==4.8 in /opt/conda/lib/python3.8/site-packages (from hydra-core&gt;=1.1.0-&gt;nemo_toolkit[all]==1.5.1) (4.8)
Requirement already satisfied: importlib-resources&lt;5.3 in /opt/conda/lib/python3.8/site-packages (from hydra-core&gt;=1.1.0-&gt;nemo_toolkit[all]==1.5.1) (5.2.3)
Requirement already satisfied: zipp&gt;=3.1.0 in /opt/conda/lib/python3.8/site-packages (from importlib-resources&lt;5.3-&gt;hydra-core&gt;=1.1.0-&gt;nemo_toolkit[all]==1.5.1) (3.7.0)
Requirement already satisfied: pip-api in /opt/conda/lib/python3.8/site-packages (from isort[requirements]&lt;5-&gt;nemo_toolkit[all]==1.5.1) (0.0.29)
Requirement already satisfied: pipreqs in /opt/conda/lib/python3.8/site-packages (from isort[requirements]&lt;5-&gt;nemo_toolkit[all]==1.5.1) (0.4.11)
Requirement already satisfied: fonttools&gt;=4.22.0 in /opt/conda/lib/python3.8/site-packages (from matplotlib-&gt;nemo_toolkit[all]==1.5.1) (4.31.2)
Requirement already satisfied: kiwisolver&gt;=1.0.1 in /opt/conda/lib/python3.8/site-packages (from matplotlib-&gt;nemo_toolkit[all]==1.5.1) (1.4.0)
Requirement already satisfied: pyparsing&gt;=2.2.1 in /opt/conda/lib/python3.8/site-packages (from matplotlib-&gt;nemo_toolkit[all]==1.5.1) (3.0.7)
Requirement already satisfied: cycler&gt;=0.10 in /opt/conda/lib/python3.8/site-packages (from matplotlib-&gt;nemo_toolkit[all]==1.5.1) (0.11.0)
Requirement already satisfied: joblib in /opt/conda/lib/python3.8/site-packages (from nltk&gt;=3.6.5-&gt;nemo_toolkit[all]==1.5.1) (1.1.0)
Requirement already satisfied: typing-extensions&gt;=3.6.2.1 in /opt/conda/lib/python3.8/site-packages (from onnx&gt;=1.7.0-&gt;nemo_toolkit[all]==1.5.1) (4.1.1)
Requirement already satisfied: six in /opt/conda/lib/python3.8/site-packages (from onnx&gt;=1.7.0-&gt;nemo_toolkit[all]==1.5.1) (1.16.0)
Requirement already satisfied: protobuf&gt;=3.12.2 in /opt/conda/lib/python3.8/site-packages (from onnx&gt;=1.7.0-&gt;nemo_toolkit[all]==1.5.1) (3.19.4)
Requirement already satisfied: pyDeprecate&lt;0.4.0,&gt;=0.3.1 in /opt/conda/lib/python3.8/site-packages (from pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (0.3.2)
Requirement already satisfied: tensorboard&gt;=2.2.0 in /opt/conda/lib/python3.8/site-packages (from pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (2.8.0)
Requirement already satisfied: fsspec[http]!=2021.06.0,&gt;=2021.05.0 in /opt/conda/lib/python3.8/site-packages (from pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (2022.2.0)
Requirement already satisfied: requests in /opt/conda/lib/python3.8/site-packages (from fsspec[http]!=2021.06.0,&gt;=2021.05.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (2.27.1)
Requirement already satisfied: aiohttp in /opt/conda/lib/python3.8/site-packages (from fsspec[http]!=2021.06.0,&gt;=2021.05.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (3.8.1)
Requirement already satisfied: werkzeug&gt;=0.11.15 in /opt/conda/lib/python3.8/site-packages (from tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (2.0.3)
Requirement already satisfied: markdown&gt;=2.6.8 in /opt/conda/lib/python3.8/site-packages (from tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (3.3.6)
Requirement already satisfied: setuptools&gt;=41.0.0 in /opt/conda/lib/python3.8/site-packages (from tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (59.5.0)
Requirement already satisfied: google-auth-oauthlib&lt;0.5,&gt;=0.4.1 in /opt/conda/lib/python3.8/site-packages (from tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (0.4.6)
Requirement already satisfied: google-auth&lt;3,&gt;=1.6.3 in /opt/conda/lib/python3.8/site-packages (from tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (2.6.2)
Requirement already satisfied: wheel&gt;=0.26 in /opt/conda/lib/python3.8/site-packages (from tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (0.37.1)
Requirement already satisfied: grpcio&gt;=1.24.3 in /opt/conda/lib/python3.8/site-packages (from tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (1.44.0)
Requirement already satisfied: tensorboard-data-server&lt;0.7.0,&gt;=0.6.0 in /opt/conda/lib/python3.8/site-packages (from tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (0.6.1)
Requirement already satisfied: tensorboard-plugin-wit&gt;=1.6.0 in /opt/conda/lib/python3.8/site-packages (from tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (1.8.1)
Requirement already satisfied: absl-py&gt;=0.4 in /opt/conda/lib/python3.8/site-packages (from tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (1.0.0)
Requirement already satisfied: cachetools&lt;6.0,&gt;=2.0.0 in /opt/conda/lib/python3.8/site-packages (from google-auth&lt;3,&gt;=1.6.3-&gt;tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (5.0.0)
Requirement already satisfied: pyasn1-modules&gt;=0.2.1 in /opt/conda/lib/python3.8/site-packages (from google-auth&lt;3,&gt;=1.6.3-&gt;tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (0.2.8)
Requirement already satisfied: rsa&lt;5,&gt;=3.1.4 in /opt/conda/lib/python3.8/site-packages (from google-auth&lt;3,&gt;=1.6.3-&gt;tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (4.8)
Requirement already satisfied: requests-oauthlib&gt;=0.7.0 in /opt/conda/lib/python3.8/site-packages (from google-auth-oauthlib&lt;0.5,&gt;=0.4.1-&gt;tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (1.3.1)
Requirement already satisfied: importlib-metadata&gt;=4.4 in /opt/conda/lib/python3.8/site-packages (from markdown&gt;=2.6.8-&gt;tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (4.11.3)
Requirement already satisfied: pyasn1&lt;0.5.0,&gt;=0.4.6 in /opt/conda/lib/python3.8/site-packages (from pyasn1-modules&gt;=0.2.1-&gt;google-auth&lt;3,&gt;=1.6.3-&gt;tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (0.4.8)
Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.8/site-packages (from requests-&gt;fsspec[http]!=2021.06.0,&gt;=2021.05.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (2.0.12)
Requirement already satisfied: certifi&gt;=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests-&gt;fsspec[http]!=2021.06.0,&gt;=2021.05.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (2021.10.8)
Requirement already satisfied: idna&lt;4,&gt;=2.5 in /opt/conda/lib/python3.8/site-packages (from requests-&gt;fsspec[http]!=2021.06.0,&gt;=2021.05.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (3.3)
Requirement already satisfied: urllib3&lt;1.27,&gt;=1.21.1 in /opt/conda/lib/python3.8/site-packages (from requests-&gt;fsspec[http]!=2021.06.0,&gt;=2021.05.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (1.26.8)
Requirement already satisfied: oauthlib&gt;=3.0.0 in /opt/conda/lib/python3.8/site-packages (from requests-oauthlib&gt;=0.7.0-&gt;google-auth-oauthlib&lt;0.5,&gt;=0.4.1-&gt;tensorboard&gt;=2.2.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (3.2.0)
Requirement already satisfied: huggingface-hub&lt;1.0,&gt;=0.1.0 in /opt/conda/lib/python3.8/site-packages (from transformers&gt;=4.0.1-&gt;nemo_toolkit[all]==1.5.1) (0.5.1)
Requirement already satisfied: tokenizers!=0.11.3,&lt;0.13,&gt;=0.11.1 in /opt/conda/lib/python3.8/site-packages (from transformers&gt;=4.0.1-&gt;nemo_toolkit[all]==1.5.1) (0.12.1)
Requirement already satisfied: filelock in /opt/conda/lib/python3.8/site-packages (from transformers&gt;=4.0.1-&gt;nemo_toolkit[all]==1.5.1) (3.6.0)
Requirement already satisfied: frozenlist&gt;=1.1.1 in /opt/conda/lib/python3.8/site-packages (from aiohttp-&gt;fsspec[http]!=2021.06.0,&gt;=2021.05.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (1.3.0)
Requirement already satisfied: yarl&lt;2.0,&gt;=1.0 in /opt/conda/lib/python3.8/site-packages (from aiohttp-&gt;fsspec[http]!=2021.06.0,&gt;=2021.05.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (1.7.2)
Requirement already satisfied: async-timeout&lt;5.0,&gt;=4.0.0a3 in /opt/conda/lib/python3.8/site-packages (from aiohttp-&gt;fsspec[http]!=2021.06.0,&gt;=2021.05.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (4.0.2)
Requirement already satisfied: multidict&lt;7.0,&gt;=4.5 in /opt/conda/lib/python3.8/site-packages (from aiohttp-&gt;fsspec[http]!=2021.06.0,&gt;=2021.05.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (6.0.2)
Requirement already satisfied: aiosignal&gt;=1.1.2 in /opt/conda/lib/python3.8/site-packages (from aiohttp-&gt;fsspec[http]!=2021.06.0,&gt;=2021.05.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (1.2.0)
Requirement already satisfied: s3transfer&lt;0.6.0,&gt;=0.5.0 in /opt/conda/lib/python3.8/site-packages (from boto3-&gt;nemo_toolkit[all]==1.5.1) (0.5.2)
Requirement already satisfied: botocore&lt;1.25.0,&gt;=1.24.45 in /opt/conda/lib/python3.8/site-packages (from boto3-&gt;nemo_toolkit[all]==1.5.1) (1.24.45)
Requirement already satisfied: jmespath&lt;2.0.0,&gt;=0.7.1 in /opt/conda/lib/python3.8/site-packages (from boto3-&gt;nemo_toolkit[all]==1.5.1) (1.0.0)
Requirement already satisfied: pybind11&gt;=2.2 in /opt/conda/lib/python3.8/site-packages (from fasttext-&gt;nemo_toolkit[all]==1.5.1) (2.9.1)
Requirement already satisfied: distance&gt;=0.1.3 in /opt/conda/lib/python3.8/site-packages (from g2p-en-&gt;nemo_toolkit[all]==1.5.1) (0.1.3)
Requirement already satisfied: beautifulsoup4 in /opt/conda/lib/python3.8/site-packages (from gdown-&gt;nemo_toolkit[all]==1.5.1) (4.10.0)
Requirement already satisfied: soupsieve&gt;1.2 in /opt/conda/lib/python3.8/site-packages (from beautifulsoup4-&gt;gdown-&gt;nemo_toolkit[all]==1.5.1) (2.3.1)
Requirement already satisfied: ipython-genutils~=0.2.0 in /opt/conda/lib/python3.8/site-packages (from ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.2.0)
Requirement already satisfied: ipython&gt;=4.0.0 in /opt/conda/lib/python3.8/site-packages (from ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (8.1.1)
Requirement already satisfied: ipykernel&gt;=4.5.1 in /opt/conda/lib/python3.8/site-packages (from ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (6.9.2)
Requirement already satisfied: jupyterlab-widgets&gt;=1.0.0 in /opt/conda/lib/python3.8/site-packages (from ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (1.1.0)
Requirement already satisfied: widgetsnbextension~=3.6.0 in /opt/conda/lib/python3.8/site-packages (from ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (3.6.0)
Requirement already satisfied: traitlets&gt;=4.3.1 in /opt/conda/lib/python3.8/site-packages (from ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (5.1.1)
Requirement already satisfied: nbformat&gt;=4.2.0 in /opt/conda/lib/python3.8/site-packages (from ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (5.2.0)
Requirement already satisfied: jupyter-client&lt;8.0 in /opt/conda/lib/python3.8/site-packages (from ipykernel&gt;=4.5.1-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (7.1.2)
Requirement already satisfied: psutil in /opt/conda/lib/python3.8/site-packages (from ipykernel&gt;=4.5.1-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (5.9.0)
Requirement already satisfied: tornado&lt;7.0,&gt;=4.2 in /opt/conda/lib/python3.8/site-packages (from ipykernel&gt;=4.5.1-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (6.1)
Requirement already satisfied: debugpy&lt;2.0,&gt;=1.0.0 in /opt/conda/lib/python3.8/site-packages (from ipykernel&gt;=4.5.1-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (1.5.1)
Requirement already satisfied: nest-asyncio in /opt/conda/lib/python3.8/site-packages (from ipykernel&gt;=4.5.1-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (1.5.4)
Requirement already satisfied: matplotlib-inline&lt;0.2.0,&gt;=0.1.0 in /opt/conda/lib/python3.8/site-packages (from ipykernel&gt;=4.5.1-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.1.3)
Requirement already satisfied: pickleshare in /opt/conda/lib/python3.8/site-packages (from ipython&gt;=4.0.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.7.5)
Requirement already satisfied: decorator in /opt/conda/lib/python3.8/site-packages (from ipython&gt;=4.0.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (5.1.1)
Requirement already satisfied: pygments&gt;=2.4.0 in /opt/conda/lib/python3.8/site-packages (from ipython&gt;=4.0.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (2.11.2)
Requirement already satisfied: stack-data in /opt/conda/lib/python3.8/site-packages (from ipython&gt;=4.0.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.2.0)
Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,&lt;3.1.0,&gt;=2.0.0 in /opt/conda/lib/python3.8/site-packages (from ipython&gt;=4.0.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (3.0.27)
Requirement already satisfied: backcall in /opt/conda/lib/python3.8/site-packages (from ipython&gt;=4.0.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.2.0)
Requirement already satisfied: jedi&gt;=0.16 in /opt/conda/lib/python3.8/site-packages (from ipython&gt;=4.0.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.18.1)
Requirement already satisfied: pexpect&gt;4.3 in /opt/conda/lib/python3.8/site-packages (from ipython&gt;=4.0.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (4.8.0)
Requirement already satisfied: parso&lt;0.9.0,&gt;=0.8.0 in /opt/conda/lib/python3.8/site-packages (from jedi&gt;=0.16-&gt;ipython&gt;=4.0.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.8.3)
Requirement already satisfied: entrypoints in /opt/conda/lib/python3.8/site-packages (from jupyter-client&lt;8.0-&gt;ipykernel&gt;=4.5.1-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.4)
Requirement already satisfied: pyzmq&gt;=13 in /opt/conda/lib/python3.8/site-packages (from jupyter-client&lt;8.0-&gt;ipykernel&gt;=4.5.1-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (22.3.0)
Requirement already satisfied: jupyter-core&gt;=4.6.0 in /opt/conda/lib/python3.8/site-packages (from jupyter-client&lt;8.0-&gt;ipykernel&gt;=4.5.1-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (4.9.2)
Requirement already satisfied: jsonschema!=2.5.0,&gt;=2.4 in /opt/conda/lib/python3.8/site-packages (from nbformat&gt;=4.2.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (4.4.0)
Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,&gt;=0.14.0 in /opt/conda/lib/python3.8/site-packages (from jsonschema!=2.5.0,&gt;=2.4-&gt;nbformat&gt;=4.2.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.18.1)
Requirement already satisfied: ptyprocess&gt;=0.5 in /opt/conda/lib/python3.8/site-packages (from pexpect&gt;4.3-&gt;ipython&gt;=4.0.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.7.0)
Requirement already satisfied: wcwidth in /opt/conda/lib/python3.8/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,&lt;3.1.0,&gt;=2.0.0-&gt;ipython&gt;=4.0.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.2.5)
Requirement already satisfied: notebook&gt;=4.4.1 in /opt/conda/lib/python3.8/site-packages (from widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (6.4.1)
Requirement already satisfied: Send2Trash&gt;=1.5.0 in /opt/conda/lib/python3.8/site-packages (from notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (1.8.0)
Requirement already satisfied: prometheus-client in /opt/conda/lib/python3.8/site-packages (from notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.13.1)
Requirement already satisfied: terminado&gt;=0.8.3 in /opt/conda/lib/python3.8/site-packages (from notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.13.3)
Requirement already satisfied: jinja2 in /opt/conda/lib/python3.8/site-packages (from notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (3.0.3)
Requirement already satisfied: nbconvert in /opt/conda/lib/python3.8/site-packages (from notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (6.4.4)
Requirement already satisfied: argon2-cffi in /opt/conda/lib/python3.8/site-packages (from notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (21.3.0)
Requirement already satisfied: argon2-cffi-bindings in /opt/conda/lib/python3.8/site-packages (from argon2-cffi-&gt;notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (21.2.0)
Requirement already satisfied: cffi&gt;=1.0.1 in /opt/conda/lib/python3.8/site-packages (from argon2-cffi-bindings-&gt;argon2-cffi-&gt;notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (1.15.0)
Requirement already satisfied: pycparser in /opt/conda/lib/python3.8/site-packages (from cffi&gt;=1.0.1-&gt;argon2-cffi-bindings-&gt;argon2-cffi-&gt;notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (2.21)
Requirement already satisfied: MarkupSafe&gt;=2.0 in /opt/conda/lib/python3.8/site-packages (from jinja2-&gt;notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (2.1.1)
Requirement already satisfied: resampy&gt;=0.2.2 in /opt/conda/lib/python3.8/site-packages (from librosa-&gt;nemo_toolkit[all]==1.5.1) (0.2.2)
Requirement already satisfied: pooch&gt;=1.0 in /opt/conda/lib/python3.8/site-packages (from librosa-&gt;nemo_toolkit[all]==1.5.1) (1.6.0)
Requirement already satisfied: audioread&gt;=2.1.5 in /opt/conda/lib/python3.8/site-packages (from librosa-&gt;nemo_toolkit[all]==1.5.1) (2.1.9)
Requirement already satisfied: llvmlite&lt;0.37,&gt;=0.36.0rc1 in /opt/conda/lib/python3.8/site-packages (from numba-&gt;nemo_toolkit[all]==1.5.1) (0.36.0)
Requirement already satisfied: threadpoolctl&gt;=2.0.0 in /opt/conda/lib/python3.8/site-packages (from scikit-learn-&gt;nemo_toolkit[all]==1.5.1) (3.1.0)
Requirement already satisfied: defusedxml in /opt/conda/lib/python3.8/site-packages (from nbconvert-&gt;notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.7.1)
Requirement already satisfied: nbclient&lt;0.6.0,&gt;=0.5.0 in /opt/conda/lib/python3.8/site-packages (from nbconvert-&gt;notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.5.13)
Requirement already satisfied: bleach in /opt/conda/lib/python3.8/site-packages (from nbconvert-&gt;notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (4.1.0)
Requirement already satisfied: mistune&lt;2,&gt;=0.8.1 in /opt/conda/lib/python3.8/site-packages (from nbconvert-&gt;notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.8.4)
Requirement already satisfied: pandocfilters&gt;=1.4.1 in /opt/conda/lib/python3.8/site-packages (from nbconvert-&gt;notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (1.5.0)
Requirement already satisfied: testpath in /opt/conda/lib/python3.8/site-packages (from nbconvert-&gt;notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.6.0)
Requirement already satisfied: jupyterlab-pygments in /opt/conda/lib/python3.8/site-packages (from nbconvert-&gt;notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.1.2)
Requirement already satisfied: webencodings in /opt/conda/lib/python3.8/site-packages (from bleach-&gt;nbconvert-&gt;notebook&gt;=4.4.1-&gt;widgetsnbextension~=3.6.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.5.1)
Requirement already satisfied: pytz&gt;=2017.3 in /opt/conda/lib/python3.8/site-packages (from pandas-&gt;nemo_toolkit[all]==1.5.1) (2021.3)
Requirement already satisfied: pip in /opt/conda/lib/python3.8/site-packages (from pip-api-&gt;isort[requirements]&lt;5-&gt;nemo_toolkit[all]==1.5.1) (21.2.4)
Requirement already satisfied: yarg in /opt/conda/lib/python3.8/site-packages (from pipreqs-&gt;isort[requirements]&lt;5-&gt;nemo_toolkit[all]==1.5.1) (0.1.9)
Requirement already satisfied: docopt in /opt/conda/lib/python3.8/site-packages (from pipreqs-&gt;isort[requirements]&lt;5-&gt;nemo_toolkit[all]==1.5.1) (0.6.2)
Requirement already satisfied: simplejson&gt;=3.8.1 in /opt/conda/lib/python3.8/site-packages (from pyannote.core-&gt;nemo_toolkit[all]==1.5.1) (3.17.6)
Requirement already satisfied: sortedcontainers&gt;=2.0.4 in /opt/conda/lib/python3.8/site-packages (from pyannote.core-&gt;nemo_toolkit[all]==1.5.1) (2.4.0)
Requirement already satisfied: tabulate&gt;=0.7.7 in /opt/conda/lib/python3.8/site-packages (from pyannote.metrics-&gt;nemo_toolkit[all]==1.5.1) (0.8.9)
Requirement already satisfied: pyannote.database&gt;=4.0.1 in /opt/conda/lib/python3.8/site-packages (from pyannote.metrics-&gt;nemo_toolkit[all]==1.5.1) (4.1.3)
Requirement already satisfied: sympy&gt;=1.1 in /opt/conda/lib/python3.8/site-packages (from pyannote.metrics-&gt;nemo_toolkit[all]==1.5.1) (1.10.1)
Requirement already satisfied: typer[all]&gt;=0.2.1 in /opt/conda/lib/python3.8/site-packages (from pyannote.database&gt;=4.0.1-&gt;pyannote.metrics-&gt;nemo_toolkit[all]==1.5.1) (0.4.0)
Requirement already satisfied: mpmath&gt;=0.19 in /opt/conda/lib/python3.8/site-packages (from sympy&gt;=1.1-&gt;pyannote.metrics-&gt;nemo_toolkit[all]==1.5.1) (1.2.1)
Requirement already satisfied: colorama&lt;0.5.0,&gt;=0.4.3 in /opt/conda/lib/python3.8/site-packages (from typer[all]&gt;=0.2.1-&gt;pyannote.database&gt;=4.0.1-&gt;pyannote.metrics-&gt;nemo_toolkit[all]==1.5.1) (0.4.4)
Requirement already satisfied: shellingham&lt;2.0.0,&gt;=1.3.0 in /opt/conda/lib/python3.8/site-packages (from typer[all]&gt;=0.2.1-&gt;pyannote.database&gt;=4.0.1-&gt;pyannote.metrics-&gt;nemo_toolkit[all]==1.5.1) (1.4.0)
Requirement already satisfied: py&gt;=1.8.2 in /opt/conda/lib/python3.8/site-packages (from pytest-&gt;nemo_toolkit[all]==1.5.1) (1.11.0)
Requirement already satisfied: iniconfig in /opt/conda/lib/python3.8/site-packages (from pytest-&gt;nemo_toolkit[all]==1.5.1) (1.1.1)
Requirement already satisfied: pluggy&lt;2.0,&gt;=0.12 in /opt/conda/lib/python3.8/site-packages (from pytest-&gt;nemo_toolkit[all]==1.5.1) (1.0.0)
Requirement already satisfied: jarowinkler&lt;1.1.0,&gt;=1.0.2 in /opt/conda/lib/python3.8/site-packages (from rapidfuzz-&gt;nemo_toolkit[all]==1.5.1) (1.0.2)
Requirement already satisfied: PySocks!=1.5.7,&gt;=1.5.6 in /opt/conda/lib/python3.8/site-packages (from requests-&gt;fsspec[http]!=2021.06.0,&gt;=2021.05.0-&gt;pytorch-lightning&gt;=1.5.0-&gt;nemo_toolkit[all]==1.5.1) (1.7.1)
Requirement already satisfied: ruamel.yaml.clib&gt;=0.2.6 in /opt/conda/lib/python3.8/site-packages (from ruamel.yaml-&gt;nemo_toolkit[all]==1.5.1) (0.2.6)
Requirement already satisfied: portalocker in /opt/conda/lib/python3.8/site-packages (from sacrebleu[ja]-&gt;nemo_toolkit[all]==1.5.1) (2.4.0)
Requirement already satisfied: ipadic&lt;2.0,&gt;=1.0 in /opt/conda/lib/python3.8/site-packages (from sacrebleu[ja]-&gt;nemo_toolkit[all]==1.5.1) (1.0.0)
Requirement already satisfied: mecab-python3==1.0.3 in /opt/conda/lib/python3.8/site-packages (from sacrebleu[ja]-&gt;nemo_toolkit[all]==1.5.1) (1.0.3)
Requirement already satisfied: sphinxcontrib-htmlhelp&gt;=2.0.0 in /opt/conda/lib/python3.8/site-packages (from sphinx-&gt;nemo_toolkit[all]==1.5.1) (2.0.0)
Requirement already satisfied: alabaster&lt;0.8,&gt;=0.7 in /opt/conda/lib/python3.8/site-packages (from sphinx-&gt;nemo_toolkit[all]==1.5.1) (0.7.12)
Requirement already satisfied: babel&gt;=1.3 in /opt/conda/lib/python3.8/site-packages (from sphinx-&gt;nemo_toolkit[all]==1.5.1) (2.9.1)
Requirement already satisfied: sphinxcontrib-serializinghtml&gt;=1.1.5 in /opt/conda/lib/python3.8/site-packages (from sphinx-&gt;nemo_toolkit[all]==1.5.1) (1.1.5)
Requirement already satisfied: sphinxcontrib-devhelp in /opt/conda/lib/python3.8/site-packages (from sphinx-&gt;nemo_toolkit[all]==1.5.1) (1.0.2)
Requirement already satisfied: sphinxcontrib-jsmath in /opt/conda/lib/python3.8/site-packages (from sphinx-&gt;nemo_toolkit[all]==1.5.1) (1.0.1)
Requirement already satisfied: sphinxcontrib-qthelp in /opt/conda/lib/python3.8/site-packages (from sphinx-&gt;nemo_toolkit[all]==1.5.1) (1.0.3)
Requirement already satisfied: snowballstemmer&gt;=1.1 in /opt/conda/lib/python3.8/site-packages (from sphinx-&gt;nemo_toolkit[all]==1.5.1) (2.2.0)
Requirement already satisfied: imagesize in /opt/conda/lib/python3.8/site-packages (from sphinx-&gt;nemo_toolkit[all]==1.5.1) (1.3.0)
Requirement already satisfied: sphinxcontrib-applehelp in /opt/conda/lib/python3.8/site-packages (from sphinx-&gt;nemo_toolkit[all]==1.5.1) (1.0.2)
Requirement already satisfied: docutils&lt;0.18,&gt;=0.14 in /opt/conda/lib/python3.8/site-packages (from sphinx-&gt;nemo_toolkit[all]==1.5.1) (0.17.1)
Requirement already satisfied: pybtex-docutils&gt;=1.0.0 in /opt/conda/lib/python3.8/site-packages (from sphinxcontrib-bibtex-&gt;nemo_toolkit[all]==1.5.1) (1.0.1)
Requirement already satisfied: pybtex&gt;=0.24 in /opt/conda/lib/python3.8/site-packages (from sphinxcontrib-bibtex-&gt;nemo_toolkit[all]==1.5.1) (0.24.0)
Requirement already satisfied: latexcodec&gt;=1.0.4 in /opt/conda/lib/python3.8/site-packages (from pybtex&gt;=0.24-&gt;sphinxcontrib-bibtex-&gt;nemo_toolkit[all]==1.5.1) (2.0.1)
Requirement already satisfied: pure-eval in /opt/conda/lib/python3.8/site-packages (from stack-data-&gt;ipython&gt;=4.0.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.2.2)
Requirement already satisfied: asttokens in /opt/conda/lib/python3.8/site-packages (from stack-data-&gt;ipython&gt;=4.0.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (2.0.5)
Requirement already satisfied: executing in /opt/conda/lib/python3.8/site-packages (from stack-data-&gt;ipython&gt;=4.0.0-&gt;ipywidgets-&gt;nemo_toolkit[all]==1.5.1) (0.8.3)
Requirement already satisfied: pathtools in /opt/conda/lib/python3.8/site-packages (from wandb-&gt;nemo_toolkit[all]==1.5.1) (0.1.2)
Requirement already satisfied: setproctitle in /opt/conda/lib/python3.8/site-packages (from wandb-&gt;nemo_toolkit[all]==1.5.1) (1.2.3)
Requirement already satisfied: GitPython&gt;=1.0.0 in /opt/conda/lib/python3.8/site-packages (from wandb-&gt;nemo_toolkit[all]==1.5.1) (3.1.27)
Requirement already satisfied: sentry-sdk&gt;=1.0.0 in /opt/conda/lib/python3.8/site-packages (from wandb-&gt;nemo_toolkit[all]==1.5.1) (1.5.10)
Requirement already satisfied: shortuuid&gt;=0.5.0 in /opt/conda/lib/python3.8/site-packages (from wandb-&gt;nemo_toolkit[all]==1.5.1) (1.0.8)
Requirement already satisfied: docker-pycreds&gt;=0.4.0 in /opt/conda/lib/python3.8/site-packages (from wandb-&gt;nemo_toolkit[all]==1.5.1) (0.4.0)
Requirement already satisfied: promise&lt;3,&gt;=2.0 in /opt/conda/lib/python3.8/site-packages (from wandb-&gt;nemo_toolkit[all]==1.5.1) (2.3)
Requirement already satisfied: gitdb&lt;5,&gt;=4.0.1 in /opt/conda/lib/python3.8/site-packages (from GitPython&gt;=1.0.0-&gt;wandb-&gt;nemo_toolkit[all]==1.5.1) (4.0.9)
Requirement already satisfied: smmap&lt;6,&gt;=3.0.1 in /opt/conda/lib/python3.8/site-packages (from gitdb&lt;5,&gt;=4.0.1-&gt;GitPython&gt;=1.0.0-&gt;wandb-&gt;nemo_toolkit[all]==1.5.1) (5.0.0)
<span class="ansi-yellow-fg">WARNING: Running pip as the &#39;root&#39; user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv</span>
</pre></div></div>
</div>
<p>## 2. Download Citrinet model</p>
<p>Next, we download a pretrained Nemo Citrinet model and convert it to a Torchscript module:</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[3]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span>import nemo
import torch

import nemo.collections.asr as nemo_asr
from nemo.core import typecheck
typecheck.set_typecheck_enabled(False)
</pre></div>
</div>
</div>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[4]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span>variant = &#39;stt_en_citrinet_256&#39;

print(f&quot;Downloading and saving {variant}...&quot;)
asr_model = nemo_asr.models.EncDecCTCModelBPE.from_pretrained(model_name=variant)
asr_model.export(f&quot;{variant}.ts&quot;)
</pre></div>
</div>
</div>
<div class="nboutput docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Downloading and saving stt_en_citrinet_256...
[NeMo I 2022-04-21 23:12:45 cloud:56] Found existing object /root/.cache/torch/NeMo/NeMo_1.5.1/stt_en_citrinet_256/91a9cc5850784b2065e8a0aa3d526fd9/stt_en_citrinet_256.nemo.
[NeMo I 2022-04-21 23:12:45 cloud:62] Re-using file from: /root/.cache/torch/NeMo/NeMo_1.5.1/stt_en_citrinet_256/91a9cc5850784b2065e8a0aa3d526fd9/stt_en_citrinet_256.nemo
[NeMo I 2022-04-21 23:12:45 common:728] Instantiating model from pre-trained checkpoint
[NeMo I 2022-04-21 23:12:46 mixins:146] Tokenizer SentencePieceTokenizer initialized with 1024 tokens
</pre></div></div>
</div>
<div class="nboutput docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area stderr docutils container">
<div class="highlight"><pre>
[NeMo W 2022-04-21 23:12:47 modelPT:130] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
    Train config :
    manifest_filepath: null
    sample_rate: 16000
    batch_size: 32
    trim_silence: true
    max_duration: 16.7
    shuffle: true
    is_tarred: false
    tarred_audio_filepaths: null
    use_start_end_token: false

[NeMo W 2022-04-21 23:12:47 modelPT:137] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s).
    Validation config :
    manifest_filepath: null
    sample_rate: 16000
    batch_size: 32
    shuffle: false
    use_start_end_token: false

[NeMo W 2022-04-21 23:12:47 modelPT:143] Please call the ModelPT.setup_test_data() or ModelPT.setup_multiple_test_data() method and provide a valid configuration file to setup the test data loader(s).
    Test config :
    manifest_filepath: null
    sample_rate: 16000
    batch_size: 32
    shuffle: false
    use_start_end_token: false

</pre></div></div>
</div>
<div class="nboutput docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
[NeMo I 2022-04-21 23:12:47 features:265] PADDING: 16
[NeMo I 2022-04-21 23:12:47 features:282] STFT using torch
</pre></div></div>
</div>
<div class="nboutput docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area stderr docutils container">
<div class="highlight"><pre>
[NeMo W 2022-04-21 23:12:47 nemo_logging:349] /opt/conda/lib/python3.8/site-packages/nemo/collections/asr/parts/preprocessing/features.py:315: FutureWarning: Pass sr=16000, n_fft=512 as keyword args. From version 0.10 passing these as positional arguments will result in an error
      librosa.filters.mel(sample_rate, self.n_fft, n_mels=nfilt, fmin=lowfreq, fmax=highfreq), dtype=torch.float

</pre></div></div>
</div>
<div class="nboutput docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
[NeMo I 2022-04-21 23:12:49 save_restore_connector:149] Model EncDecCTCModelBPE was successfully restored from /root/.cache/torch/NeMo/NeMo_1.5.1/stt_en_citrinet_256/91a9cc5850784b2065e8a0aa3d526fd9/stt_en_citrinet_256.nemo.
</pre></div></div>
</div>
<div class="nboutput docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area stderr docutils container">
<div class="highlight"><pre>
[NeMo W 2022-04-21 23:12:49 export_utils:198] Swapped 0 modules
[NeMo W 2022-04-21 23:12:49 conv_asr:73] Turned off 235 masked convolutions
[NeMo W 2022-04-21 23:12:49 export_utils:198] Swapped 0 modules
[NeMo W 2022-04-21 23:12:50 nemo_logging:349] /opt/conda/lib/python3.8/site-packages/torch/jit/_trace.py:916: UserWarning: `optimize` is deprecated and has no effect. Use `with torch.jit.optimized_execution() instead
      warnings.warn(

[NeMo W 2022-04-21 23:12:50 nemo_logging:349] /opt/conda/lib/python3.8/site-packages/torch/_jit_internal.py:668: LightningDeprecationWarning: The `LightningModule.model_size` property was deprecated in v1.5 and will be removed in v1.7. Please use the `pytorch_lightning.utilities.memory.get_model_size_mb`.
      if hasattr(mod, name):

[NeMo W 2022-04-21 23:12:50 nemo_logging:349] /opt/conda/lib/python3.8/site-packages/torch/_jit_internal.py:669: LightningDeprecationWarning: The `LightningModule.model_size` property was deprecated in v1.5 and will be removed in v1.7. Please use the `pytorch_lightning.utilities.memory.get_model_size_mb`.
      item = getattr(mod, name)

[NeMo W 2022-04-21 23:12:50 nemo_logging:349] /opt/conda/lib/python3.8/site-packages/torch/_jit_internal.py:668: LightningDeprecationWarning: `LightningModule.use_amp` was deprecated in v1.6 and will be removed in v1.8. Please use `Trainer.amp_backend`.
      if hasattr(mod, name):

[NeMo W 2022-04-21 23:12:50 nemo_logging:349] /opt/conda/lib/python3.8/site-packages/torch/_jit_internal.py:669: LightningDeprecationWarning: `LightningModule.use_amp` was deprecated in v1.6 and will be removed in v1.8. Please use `Trainer.amp_backend`.
      item = getattr(mod, name)

</pre></div></div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[4]:
</pre></div>
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
([&#39;stt_en_citrinet_256.ts&#39;],
 [&#39;nemo.collections.asr.models.ctc_bpe_models.EncDecCTCModelBPE exported to ONNX&#39;])
</pre></div></div>
</div>
<section id="Benchmark-utility">
<h3>Benchmark utility<a class="headerlink" href="#Benchmark-utility" title="Permalink to this headline">¶</a></h3>
<p>Let us define a helper benchmarking function, then benchmark the original Pytorch model.</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[5]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span>from __future__ import print_function
from __future__ import absolute_import
from __future__ import division

import argparse
import timeit
import numpy as np
import torch
import torch_tensorrt as trtorch
import torch.backends.cudnn as cudnn

def benchmark(model, input_tensor, num_loops, model_name, batch_size):
    def timeGraph(model, input_tensor, num_loops):
        print(&quot;Warm up ...&quot;)
        with torch.no_grad():
            for _ in range(20):
                features = model(input_tensor)

        torch.cuda.synchronize()
        print(&quot;Start timing ...&quot;)
        timings = []
        with torch.no_grad():
            for i in range(num_loops):
                start_time = timeit.default_timer()
                features = model(input_tensor)
                torch.cuda.synchronize()
                end_time = timeit.default_timer()
                timings.append(end_time - start_time)
                # print(&quot;Iteration {}: {:.6f} s&quot;.format(i, end_time - start_time))
        return timings
    def printStats(graphName, timings, batch_size):
        times = np.array(timings)
        steps = len(times)
        speeds = batch_size / times
        time_mean = np.mean(times)
        time_med = np.median(times)
        time_99th = np.percentile(times, 99)
        time_std = np.std(times, ddof=0)
        speed_mean = np.mean(speeds)
        speed_med = np.median(speeds)
        msg = (&quot;\n%s =================================\n&quot;
                &quot;batch size=%d, num iterations=%d\n&quot;
                &quot;  Median samples/s: %.1f, mean: %.1f\n&quot;
                &quot;  Median latency (s): %.6f, mean: %.6f, 99th_p: %.6f, std_dev: %.6f\n&quot;
                ) % (graphName,
                    batch_size, steps,
                    speed_med, speed_mean,
                    time_med, time_mean, time_99th, time_std)
        print(msg)
    timings = timeGraph(model, input_tensor, num_loops)
    printStats(model_name, timings, batch_size)

precisions_str = &#39;fp32&#39; # Precision (default=fp32, fp16)
variant = &#39;stt_en_citrinet_256&#39; # Nemo Citrinet variant
batch_sizes = [1, 8, 32, 128] # Batch sizes (default=1,8,32,128)
trt = False # If True, infer with Torch-TensorRT engine. Else, infer with Pytorch model.
precision = torch.float32 if precisions_str ==&#39;fp32&#39; else torch.float16

for batch_size in batch_sizes:
    if trt:
        model_name = f&quot;{variant}_bs{batch_size}_{precision}.torch-tensorrt&quot;
    else:
        model_name = f&quot;{variant}.ts&quot;

    print(f&quot;Loading model: {model_name}&quot;)
    # Load traced model to CPU first
    model = torch.jit.load(model_name).cuda()
    cudnn.benchmark = True
    # Create random input tensor of certain size
    torch.manual_seed(12345)
    input_shape=(batch_size, 80, 1488)
    input_tensor = torch.randn(input_shape).cuda()

    # Timing graph inference
    benchmark(model, input_tensor, 50, model_name, batch_size)
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Loading model: stt_en_citrinet_256.ts
Warm up ...
Start timing ...

stt_en_citrinet_256.ts =================================
batch size=1, num iterations=50
  Median samples/s: 102.0, mean: 102.0
  Median latency (s): 0.009802, mean: 0.009803, 99th_p: 0.009836, std_dev: 0.000014

Loading model: stt_en_citrinet_256.ts
Warm up ...
Start timing ...

stt_en_citrinet_256.ts =================================
batch size=8, num iterations=50
  Median samples/s: 429.1, mean: 429.1
  Median latency (s): 0.018642, mean: 0.018643, 99th_p: 0.018670, std_dev: 0.000014

Loading model: stt_en_citrinet_256.ts
Warm up ...
Start timing ...

stt_en_citrinet_256.ts =================================
batch size=32, num iterations=50
  Median samples/s: 551.3, mean: 551.2
  Median latency (s): 0.058047, mean: 0.058053, 99th_p: 0.058375, std_dev: 0.000106

Loading model: stt_en_citrinet_256.ts
Warm up ...
Start timing ...

stt_en_citrinet_256.ts =================================
batch size=128, num iterations=50
  Median samples/s: 594.1, mean: 594.1
  Median latency (s): 0.215434, mean: 0.215446, 99th_p: 0.215806, std_dev: 0.000116

</pre></div></div>
</div>
<p>Confirming the GPU we are using here:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[6]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span>!nvidia-smi
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Thu Apr 21 23:13:32 2022
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 510.47.03    Driver Version: 510.47.03    CUDA Version: 11.6     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  NVIDIA TITAN V      On   | 00000000:17:00.0 Off |                  N/A |
| 38%   55C    P2    42W / 250W |   2462MiB / 12288MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+
|   1  NVIDIA TITAN V      On   | 00000000:65:00.0 Off |                  N/A |
| 28%   39C    P8    26W / 250W |    112MiB / 12288MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|    0   N/A  N/A      3909      G                                       4MiB |
|    0   N/A  N/A      6047      C                                    2453MiB |
|    1   N/A  N/A      3909      G                                      39MiB |
|    1   N/A  N/A      4161      G                                      67MiB |
+-----------------------------------------------------------------------------+
</pre></div></div>
</div>
<p>## 3. Create Torch-TensorRT modules</p>
<p>In this step, we optimize the Citrinet Torchscript module with Torch-TensorRT with various precisions and batch sizes.</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[10]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span>import torch
import torch.nn as nn
import torch_tensorrt as torchtrt
import argparse

variant = &quot;stt_en_citrinet_256&quot;
precisions = [torch.float, torch.half]
batch_sizes = [1,8,32,128]

model = torch.jit.load(f&quot;{variant}.ts&quot;)

for precision in precisions:
    for batch_size in batch_sizes:
        compile_settings = {
            &quot;inputs&quot;: [torchtrt.Input(shape=[batch_size, 80, 1488])],
            &quot;enabled_precisions&quot;: {precision},
            &quot;workspace_size&quot;: 2000000000,
            &quot;truncate_long_and_double&quot;: True,
        }
        print(f&quot;Generating Torchscript-TensorRT module for batchsize {batch_size} precision {precision}&quot;)
        trt_ts_module = torchtrt.compile(model, **compile_settings)
        torch.jit.save(trt_ts_module, f&quot;{variant}_bs{batch_size}_{precision}.torch-tensorrt&quot;)
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Generating Torchscript-TensorRT module for batchsize 1 precision torch.float32
Generating Torchscript-TensorRT module for batchsize 8 precision torch.float32
Generating Torchscript-TensorRT module for batchsize 32 precision torch.float32
Generating Torchscript-TensorRT module for batchsize 128 precision torch.float32
Generating Torchscript-TensorRT module for batchsize 1 precision torch.float16
Generating Torchscript-TensorRT module for batchsize 8 precision torch.float16
Generating Torchscript-TensorRT module for batchsize 32 precision torch.float16
Generating Torchscript-TensorRT module for batchsize 128 precision torch.float16
</pre></div></div>
</div>
<p>## 4. Benchmark Torch-TensorRT models</p>
<p>Finally, we are ready to benchmark the Torch-TensorRT optimized Citrinet models.</p>
</section>
<section id="FP32-(single-precision)">
<h3>FP32 (single precision)<a class="headerlink" href="#FP32-(single-precision)" title="Permalink to this headline">¶</a></h3>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[13]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span>precisions_str = &#39;fp32&#39; # Precision (default=fp32, fp16)
batch_sizes = [1, 8, 32, 128] # Batch sizes (default=1,8,32,128)
precision = torch.float32 if precisions_str ==&#39;fp32&#39; else torch.float16
trt = True

for batch_size in batch_sizes:
    if trt:
        model_name = f&quot;{variant}_bs{batch_size}_{precision}.torch-tensorrt&quot;
    else:
        model_name = f&quot;{variant}.ts&quot;

    print(f&quot;Loading model: {model_name}&quot;)
    # Load traced model to CPU first
    model = torch.jit.load(model_name).cuda()
    cudnn.benchmark = True
    # Create random input tensor of certain size
    torch.manual_seed(12345)
    input_shape=(batch_size, 80, 1488)
    input_tensor = torch.randn(input_shape).cuda()

    # Timing graph inference
    benchmark(model, input_tensor, 50, model_name, batch_size)
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Loading model: stt_en_citrinet_256_bs1_torch.float32.torch-tensorrt
Warm up ...
Start timing ...

stt_en_citrinet_256_bs1_torch.float32.torch-tensorrt =================================
batch size=1, num iterations=50
  Median samples/s: 242.2, mean: 218.0
  Median latency (s): 0.004128, mean: 0.004825, 99th_p: 0.008071, std_dev: 0.001270

Loading model: stt_en_citrinet_256_bs8_torch.float32.torch-tensorrt
Warm up ...
Start timing ...

stt_en_citrinet_256_bs8_torch.float32.torch-tensorrt =================================
batch size=8, num iterations=50
  Median samples/s: 729.9, mean: 709.0
  Median latency (s): 0.010961, mean: 0.011388, 99th_p: 0.016114, std_dev: 0.001256

Loading model: stt_en_citrinet_256_bs32_torch.float32.torch-tensorrt
Warm up ...
Start timing ...

stt_en_citrinet_256_bs32_torch.float32.torch-tensorrt =================================
batch size=32, num iterations=50
  Median samples/s: 955.6, mean: 953.4
  Median latency (s): 0.033488, mean: 0.033572, 99th_p: 0.035722, std_dev: 0.000545

Loading model: stt_en_citrinet_256_bs128_torch.float32.torch-tensorrt
Warm up ...
Start timing ...

stt_en_citrinet_256_bs128_torch.float32.torch-tensorrt =================================
batch size=128, num iterations=50
  Median samples/s: 1065.8, mean: 1069.4
  Median latency (s): 0.120097, mean: 0.119708, 99th_p: 0.121618, std_dev: 0.001260

</pre></div></div>
</div>
</section>
<section id="FP16-(half-precision)">
<h3>FP16 (half precision)<a class="headerlink" href="#FP16-(half-precision)" title="Permalink to this headline">¶</a></h3>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[14]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span>precisions_str = &#39;fp16&#39; # Precision (default=fp32, fp16)
batch_sizes = [1, 8, 32, 128] # Batch sizes (default=1,8,32,128)
precision = torch.float32 if precisions_str ==&#39;fp32&#39; else torch.float16

for batch_size in batch_sizes:
    if trt:
        model_name = f&quot;{variant}_bs{batch_size}_{precision}.torch-tensorrt&quot;
    else:
        model_name = f&quot;{variant}.ts&quot;

    print(f&quot;Loading model: {model_name}&quot;)
    # Load traced model to CPU first
    model = torch.jit.load(model_name).cuda()
    cudnn.benchmark = True
    # Create random input tensor of certain size
    torch.manual_seed(12345)
    input_shape=(batch_size, 80, 1488)
    input_tensor = torch.randn(input_shape).cuda()

    # Timing graph inference
    benchmark(model, input_tensor, 50, model_name, batch_size)
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Loading model: stt_en_citrinet_256_bs1_torch.float16.torch-tensorrt
Warm up ...
Start timing ...

stt_en_citrinet_256_bs1_torch.float16.torch-tensorrt =================================
batch size=1, num iterations=50
  Median samples/s: 288.9, mean: 272.9
  Median latency (s): 0.003462, mean: 0.003774, 99th_p: 0.006846, std_dev: 0.000820

Loading model: stt_en_citrinet_256_bs8_torch.float16.torch-tensorrt
Warm up ...
Start timing ...

stt_en_citrinet_256_bs8_torch.float16.torch-tensorrt =================================
batch size=8, num iterations=50
  Median samples/s: 1201.0, mean: 1190.9
  Median latency (s): 0.006661, mean: 0.006733, 99th_p: 0.008453, std_dev: 0.000368

Loading model: stt_en_citrinet_256_bs32_torch.float16.torch-tensorrt
Warm up ...
Start timing ...

stt_en_citrinet_256_bs32_torch.float16.torch-tensorrt =================================
batch size=32, num iterations=50
  Median samples/s: 1538.2, mean: 1516.4
  Median latency (s): 0.020804, mean: 0.021143, 99th_p: 0.024492, std_dev: 0.000973

Loading model: stt_en_citrinet_256_bs128_torch.float16.torch-tensorrt
Warm up ...
Start timing ...

stt_en_citrinet_256_bs128_torch.float16.torch-tensorrt =================================
batch size=128, num iterations=50
  Median samples/s: 1792.0, mean: 1777.0
  Median latency (s): 0.071428, mean: 0.072057, 99th_p: 0.076796, std_dev: 0.001351

</pre></div></div>
</div>
<p>## 5. Conclusion</p>
<p>In this notebook, we have walked through the complete process of optimizing the Citrinet model with Torch-TensorRT. On an A100 GPU, with Torch-TensorRT, we observe a speedup of ~<strong>2.4X</strong> with FP32, and ~<strong>2.9X</strong> with FP16 at batchsize of 128.</p>
</section>
<section id="What’s-next">
<h3>What’s next<a class="headerlink" href="#What’s-next" title="Permalink to this headline">¶</a></h3>
<p>Now it’s time to try Torch-TensorRT on your own model. Fill out issues at <a class="reference external" href="https://github.com/NVIDIA/Torch-TensorRT">https://github.com/NVIDIA/Torch-TensorRT</a>. Your involvement will help future development of Torch-TensorRT.</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[ ]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span>
</pre></div>
</div>
</div>
</section>
</section>
</section>


             </article>

            </div>
            <footer>




    <hr>



  <div role="contentinfo">
    <p>
        &copy; Copyright 2021, NVIDIA Corporation.

    </p>
  </div>

      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>


</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              <ul>
<li><a class="reference internal" href="#">Torch-TensorRT Getting Started - CitriNet</a><ul>
<li><a class="reference internal" href="#Overview">Overview</a><ul>
<li><a class="reference internal" href="#Learning-objectives">Learning objectives</a></li>
</ul>
</li>
<li><a class="reference internal" href="#Content">Content</a><ul>
<li><a class="reference internal" href="#Benchmark-utility">Benchmark utility</a></li>
<li><a class="reference internal" href="#FP32-(single-precision)">FP32 (single precision)</a></li>
<li><a class="reference internal" href="#FP16-(half-precision)">FP16 (half precision)</a></li>
<li><a class="reference internal" href="#What’s-next">What’s next</a></li>
</ul>
</li>
</ul>
</li>
</ul>

            </div>
          </div>
        </div>
      </section>
    </div>







       <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
         <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
         <script src="../_static/jquery.js"></script>
         <script src="../_static/underscore.js"></script>
         <script src="../_static/doctools.js"></script>
         <script src="../_static/collapsible-lists/js/CollapsibleLists.compressed.js"></script>
         <script src="../_static/collapsible-lists/js/apply-collapsible-lists.js"></script>
         <script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
         <script>window.MathJax = {"tex": {"inlineMath": [["$", "$"], ["\\(", "\\)"]], "processEscapes": true}, "options": {"ignoreHtmlClass": "tex2jax_ignore|mathjax_ignore|document", "processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
         <script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>




  <script type="text/javascript" src="../_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="../_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/resources">Resources</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.org/assets/brand-guidelines/PyTorch-Brand-Guidelines.pdf" target="_blank">Brand Guidelines</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title">Stay up to date</li>
            <li><a href="https://www.facebook.com/pytorch" target="_blank">Facebook</a></li>
            <li><a href="https://twitter.com/pytorch" target="_blank">Twitter</a></li>
            <li><a href="https://www.youtube.com/pytorch" target="_blank">YouTube</a></li>
            <li><a href="https://www.linkedin.com/company/pytorch" target="_blank">LinkedIn</a></li>
          </ul>
          </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title">PyTorch Podcasts</li>
            <li><a href="https://open.spotify.com/show/6UzHKeiy368jKfQMKKvJY5" target="_blank">Spotify</a></li>
            <li><a href="https://podcasts.apple.com/us/podcast/pytorch-developer-podcast/id1566080008" target="_blank">Apple</a></li>
            <li><a href="https://www.google.com/podcasts?feed=aHR0cHM6Ly9mZWVkcy5zaW1wbGVjYXN0LmNvbS9PQjVGa0lsOA%3D%3D" target="_blank">Google</a></li>
            <li><a href="https://music.amazon.com/podcasts/7a4e6f0e-26c2-49e9-a478-41bd244197d0/PyTorch-Developer-Podcast?" target="_blank">Amazon</a></li>
          </ul>
         </div>
        </div>

        <div class="privacy-policy">
          <ul>
            <li class="privacy-policy-links"><a href="https://www.linuxfoundation.org/terms/" target="_blank">Terms</a></li>
            <li class="privacy-policy-links">|</li>
            <li class="privacy-policy-links"><a href="https://www.linuxfoundation.org/privacy-policy/" target="_blank">Privacy</a></li>
          </ul>
        </div>
        <div class="copyright">
        <p>© Copyright The Linux Foundation. The PyTorch Foundation is a project of The Linux Foundation.
          For web site terms of use, trademark policy and other policies applicable to The PyTorch Foundation please see
          <a href="www.linuxfoundation.org/policies/">www.linuxfoundation.org/policies/</a>. The PyTorch Foundation supports the PyTorch open source
          project, which has been established as PyTorch Project a Series of LF Projects, LLC. For policies applicable to the PyTorch Project a Series of LF Projects, LLC,
          please see <a href="www.lfprojects.org/policies/">www.lfprojects.org/policies/</a>.</p>
      </div>
     </div>

  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="../_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/hub">PyTorch Hub</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="resources-mobile-menu-title">
            Docs
          </li>

          <ul class="resources-mobile-menu-items">
            <li>
              <a href="https://pytorch.org/docs/stable/index.html">PyTorch</a>
            </li>

            <li>
              <a href="https://pytorch.org/audio/stable/index.html">torchaudio</a>
            </li>

            <li>
              <a href="https://pytorch.org/text/stable/index.html">torchtext</a>
            </li>

            <li>
              <a href="https://pytorch.org/vision/stable/index.html">torchvision</a>
            </li>

            <li>
              <a href="https://pytorch.org/serve/">TorchServe</a>
            </li>

            <li>
              <a href="https://pytorch.org/torchx/">TorchX</a>
            </li>

            <li>
              <a href="https://pytorch.org/xla">PyTorch on XLA Devices</a>
            </li>
          </ul>

          <li class="resources-mobile-menu-title">
            Resources
          </li>

          <ul class="resources-mobile-menu-items">
            <li>
              <a href="https://pytorch.org/resources">Developer Resources</a>
            </li>

            <li>
              <a href="https://pytorch.org/features">About</a>
            </li>

            <li>
              <a href="https://pytorch.org/hub">Models (Beta)</a>
            </li>

            <li>
              <a href="https://pytorch.org/#community-module">Community</a>
            </li>

            <li>
              <a href="https://discuss.pytorch.org/">Forums</a>
            </li>
          </ul>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="../_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>